
import java.util.*;
import java.net.*;
import java.io.*;
import org.json.*;


/**
 * The prober object should be used for a singular classification scheme which
 * in this implementation is predefined and built in.
 * @author Reuben Doetsch
 */
public class Prober
{
    //String used for BOSS API query constrution
    private static final String BOSS_URI= "http://boss.yahooapis.com/ysearch/web/v1/";
    private static final String APP_URI="?appid=";
    private static final String FORMAT_URI= "&format=json";
    private static final String SITE_URL="&sites=";
    private static final String TEXT_EXT=".txt";

    //the boss id retrived from www.yahoo.com
    private String yahooBossID;
    //the root of the classification system
    private Node rootNode;
    //list of urls linked to category names e.g. root, health
    private Hashtable<String,Set<String>> listOfURLs;
    //a hash mapping category names to list of indexes
    private Hashtable<String, Hashtable<String, Integer>> categoriesToContentSummaries;
    //the database URL
    private String databaseURL;
    //a storage of categories to nodes
    Hashtable<String,Node> categoriesToNodes;
  
    /**
     * This is a constructer for the Querier wih the specfied appID and classification
     * system.
     * @param appId the Yahoo BOSS API
     * @param dbURL the URL of the database to be categorized
     */
    public Prober(String appId, String dbURL)
    {
        Debug.print("Starting prober with appId" + appId + " and database URL "+ dbURL);
        yahooBossID=appId;
        databaseURL=dbURL;
        listOfURLs=new Hashtable<String, Set<String>>();

        Debug.print("Reading in categorization tree");
        rootNode=readInCategorizationTree(); 
    }
    
    /**
     * This runs the prober with the spefied coverage and specficity
     * @param ec the coverage
     * @param es the specificity
     */
    public void runProber(int ec, double es)
    {
        System.out.println("Classifying...");
        //Classify the database
        ArrayList<Node> classifaction=classify(rootNode, ec,es, 1);

        //Print out the node classifaction
        String temp="";
        for(Node classifactionNode:classifaction)
        {
            Node t=classifactionNode;
            while(t.parent!=null){
                temp= t.categoryName + "\\" + temp;
                t=t.parent;
            }
        }

        System.out.println("\n\nClassification:");
        System.out.println("Root\\" + temp);

        //output content summaries
        createContentSummaries();

    }

    /**
     * This classifies a databases with categorization node n, database
     * corresponding to the specfic database url. It has ___ of ec and
     * ___ of es. The _______ is parentes
     * database string databaseURL,
     * @param n This is the node being classfied
     * @param ec the eCoverage value
     * @param es the eSpecficty value
     * @param parentes the parents eSpecficity
     * @return the recursive list of nodes
     */
    public ArrayList<Node> classify(Node n, int ec, double es, double parentes)
    {
        try
        {
        System.out.println("Classifying "+n.categoryName);
        //the result array
        ArrayList<Node> result=new ArrayList();

        //if leaf node then add that as a result and return
        if(n.isLeafNode()){
           result.add(n);
           System.out.println("Classification is done with category " + n.categoryName);
           return result;
        }

        //Creates new list of urls for category name
        listOfURLs.put(n.categoryName,new HashSet());

        //gets the eCoverages of all the childern and compute total of
        //all e coverages
        int sumOfECoverages=0; //sum of e coverages
        int[] eCoverages=new int[n.childern.size()];//array storing all eCoverages
        //go though each child and populate all their ecoverages and add to sum of
        //total ecoverages
        for(int i=0;i<n.childern.size();i++)
        {
            int sizeQuery = findECoverage(n.childern.get(i));
            eCoverages[i]=sizeQuery;
            sumOfECoverages+=sizeQuery;
            Debug.print("Size of "+ n.childern.get(i).categoryName+ " is "+sizeQuery);
        }

        Debug.print("ECoverage");
     //   printArray(eCoverages);

        //keep track of all specficies for all childern
        double[] eSpec=new double[n.childern.size()];

        for(int i=0;i<n.childern.size();i++)
        {
            eSpec[i]=(parentes * eCoverages[i])/sumOfECoverages;
        }

        Debug.print("ESpecs");

        //for each child if the specficity is higher than specfied
        //and the coverage is higher than threshold, then classify with that
        //root node and add to result
        for(int i=0;i<n.childern.size();i++)
        {
            System.out.println("Specificity for category: "+n.childern.get(i).categoryName + " " + eSpec[i]);
            System.out.println("Coverage for category: " + n.childern.get(i).categoryName + " " + eCoverages[i]);
            if(eSpec[i]>=es && eCoverages[i]>=ec)
            {
                Debug.print("Going inside "+ n.childern.get(i));
                ArrayList<Node> aln= classify(n.childern.get(i),ec,es,eSpec[i]);
                Debug.print("Now adding "+aln + " to orginal dataset ");
                result.addAll(aln);
            }
        }

        //if the result is nothing then add this node and return result
        if(result.isEmpty())
        {
            result.add(n);
            Debug.print("REturning "+result.toString());
            return result;
        }

        Debug.print("REturning "+result.toString());
        return result;
        }
        catch(Exception e)
        {
            e.printStackTrace();
            System.out.println(e.getMessage());
            System.out.println("Something wrong");
        }
        return null;
    }

    /**
     * This finds the e coverage of a node n.
     * @param n the node to find the coverage
     * @return the total coverage
     */
    public int findECoverage(Node n)
    {
        Debug.print("Finding ecoverage for " +n.categoryName + " with "+ n.parent.categoryName);
        int size=0;

        for(String probingString: n.probingQueries)
        {
            Debug.print("Probing " + probingString);
            int totalHits=this.getSizeOfQuery(probingString);
            Debug.print("Total hits "+ totalHits);
            size+=totalHits;
          //  if(!listOfURLs.containsKey(n.categoryName))
            //        listOfURLs.put(n.categoryName, new HashSet<String>());
            
            listOfURLs.get(n.parent.categoryName).addAll(getTopFourResults(probingString));
        }
        return size;
    }
 
    /******************Dealing CONTENT SUMMARY *****************
    //All deal with part two of assignment


     /**
     * This returns a hashtable of a dictionary with the certian urls
     * @param listOfURLs the list of urls to run the inquiries
     * @return a hashtable with the frequencies in the documents
     */
    public Hashtable<String,Integer> createDictionary(Set<String> listOfURLs)
    {
            Hashtable<String, Integer> contextSummary=new Hashtable<String,Integer>();
            for(String url:listOfURLs)
            {
                System.out.println("Getting page "+ url);
                Set<String> wordSet=LynxReturner.runLynx(url);
                for(String word:wordSet)
                {
                    if(!contextSummary.containsKey(word))
                        contextSummary.put(word, 0);
                    Integer tempInt=contextSummary.get(word);
                    contextSummary.put(word, ++tempInt);
                }

            }
            return contextSummary;
    }

    /**
     * This creates content summaries for all of the
     */
    public void createContentSummaries()
    {

        //the listOfURLs.keySet() is just a list of the nodes that we used probers
        //to find
        for(String category: listOfURLs.keySet())
        {
            createContentSummary(category);
        }
    }

    /**
     * This creates a content summary for the certian category
     * @param category the category to create summary for.
     */
    public void createContentSummary(String category)
    {
        System.out.println("Creating content summary for "+ category);
        Set totalURLs=new HashSet();
        
        if(category.equalsIgnoreCase("Root"))
        {   
            for(String cat: listOfURLs.keySet())
                totalURLs.addAll(listOfURLs.get(cat));
        }
        else
            totalURLs=listOfURLs.get(category);

       printDictionary(createDictionary(totalURLs), category+"-"+databaseURL+TEXT_EXT );
    }

    /**
     * This prints out a dictionary in alphabetic order.
     * @param dict the dictionary
     * @param fileName the name of the file to write to
     */
    public void printDictionary(Hashtable<String, Integer> dict, String fileName)
    {
        try
        {
            BufferedWriter buffWriter=new BufferedWriter(new FileWriter(fileName));
            Object[] dictEntries=dict.keySet().toArray();

            Arrays.sort(dictEntries);
            for(Object sortedWord: dictEntries)
            {
                buffWriter.write(sortedWord +"#"+dict.get(sortedWord));
                buffWriter.write("\n");
            }
            buffWriter.close();
        } catch(Exception e)
        {
            e.printStackTrace();
            System.out.println(e.getMessage());
            System.out.println("Buffered Writer not working");
        }

    }

    /********** JSON Functions **************/
    //All of these interface with json.

    /**
     * Get the size of the query (num results) in the database with the
     * url dbURL (e.g. java.sun.com and the query query
     * @param query the query
     **/
    public int getSizeOfQuery(String query)
    {
        try
        {
            String jsonContent=getContent(this.getURLConnectionForQuery(query));
            JSONObject json=new JSONObject(jsonContent);
            String a=(String)json.getJSONObject("ysearchresponse").get("totalhits");
            Debug.print(a);
            return Integer.parseInt(a);
        }catch (Exception e)
        {
            System.out.println("Can not find size of query. Check connection");
        }
        return 0;
    }

     /**
     * This returns the Query URL
     * @param queryString this is the queryString used to create the URl
     * @return this returns the query url used to as the query to yahoo boss with the
     * specifed queryString
     */
    public String getQueryURL(String queryString)
    {
        String queryURL=BOSS_URI+queryString+APP_URI+yahooBossID+FORMAT_URI+SITE_URL+databaseURL;
        Debug.print("Returning" +queryURL);
        return queryURL;
    }

    /**
     * Gets the top four results from the query
     * @param query the query
     * @return returns the top four results in an arraylist
     */
    public ArrayList<String> getTopFourResults(String query)
    {
        try
        {
        String jsonContent=getContent(this.getURLConnectionForQuery(query));

        JSONObject json=new JSONObject(jsonContent);
        ArrayList<String> listOfUrls = new ArrayList<String>();
    

        int num= Integer.parseInt((String)json.getJSONObject("ysearchresponse").get("totalhits"));

        if(num > 4)
            num=4;

        if(num!=0)
        {
            JSONArray ja = json.getJSONObject("ysearchresponse").getJSONArray("resultset_web");
           

            for (int i = 0; i < num; i++) {
                JSONObject j = ja.getJSONObject(i);
                 Debug.print("Adding url "+ j.getString("url"));
                 listOfUrls.add(j.getString("url"));
            }
        }
        return listOfUrls;
        }
        catch(Exception e)
        {
            e.printStackTrace();
        }
        return null;
    }

    /** READING IN CATEGORIZATION TABLES **/

    /**
     * This reads in the categorization tables.
     * @return the root Node of the categorization system
     */
    public Node readInCategorizationTree()
    {
        Node rootNode=new Node("Root");
        categoriesToNodes=new Hashtable<String, Node>();
        categoriesToNodes.put("Root", rootNode);
        readInCategorizationTree(rootNode);
        return rootNode;
    }
    
    /**
     * Private function that recursively reads in categorization tables. This
     * function never needs to be called and should instead use @see readInCategorizationTree
     * @param parentNode the parent node
     */
    private void readInCategorizationTree(Node parentNode)
    {
       try
       {
        Debug.print("Opening: " + parentNode.categoryName.toLowerCase()+".txt");
        BufferedReader reader= new BufferedReader(new FileReader(parentNode.categoryName.toLowerCase()+".txt"));     
        ArrayList<Node> currentLevelNodes=new ArrayList<Node>();
        while(reader.ready())
        {
            String line=reader.readLine();
            String[] lineTokens=line.split(" ");
            String category=lineTokens[0];
            if(!categoriesToNodes.containsKey(lineTokens[0]))
            {
                Debug.print("Adding category "+ category);
                Node tempNode= new Node(category, parentNode);
                tempNode.parent=parentNode;
                categoriesToNodes.put(category, tempNode);
                currentLevelNodes.add(tempNode);
                parentNode.childern.add(tempNode);
            }
            categoriesToNodes.get(category).probingQueries.add(this.toHTML(concatStrings(lineTokens, 1)));
        }

        for(Node n: currentLevelNodes)
            readInCategorizationTree(n);

        }
        catch (FileNotFoundException e){
            System.out.println("If " +  parentNode.categoryName + " has subcategories then please add "+ parentNode.categoryName+TEXT_EXT+" to correct directory. See README");
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
    }

    /**** URL UTILITY ************/
    //These are all url utility functions

    /**
     * Creates a connection to the url specfied and returns string
     * representation of content
     * @param c the URL connection
     * @return the URL connection string
     */
    public String getContent(URLConnection c)
    {
        try
        {
            String line;
            StringBuilder builder = new StringBuilder();
            BufferedReader reader = new BufferedReader(new InputStreamReader(c.getInputStream()));
            while((line = reader.readLine()) != null) {
                builder.append(line);
            }
            return builder.toString();
        }
        catch(Exception e)
        {
            System.out.println("getContent not working");
            e.printStackTrace();
        }
        return "";
    }

       /**
     * This gets the URL Connection for the query, a utility function
     * @param query the query
     * @return the URLConnection
     */
    public URLConnection getURLConnectionForQuery(String query)
    {
        try{
         return new URL(getQueryURL(query)).openConnection();
        }
        catch(IOException ioe)
        {
            System.out.println("io exception");
        }
        return null;
    }

    /*** UTILITY FUNCTIONS ***/
    //All other utility functions

    /**
     * This concates all strings in an array after a certain index
     * @param lines the array of lines of the string
     * @param indexToConcatFrom the index to start at
     * @return the concatenated string.
     */
    public static String concatStrings(String[] lines, int indexToConcatFrom)
    {
        String temp="";
        for(int i=indexToConcatFrom;i<lines.length;i++)
            temp+=lines[i]+ " ";

        return temp.substring(0,temp.length()-1);
    }

    /**
     * Format a query for display purposes.
     * @param q the query to be formatted
     * @return the formatted query
     */
    public String formatQuery(String q){
        String f;
        f = q.replaceAll("%20", " ");
        return f;
    }

    public String toHTML(String q)
    {
        String f;
        f = q.replaceAll(" ", "%20");
        return f;
    }

    /**
     * Generic method to print out a array of anything spaced.
     * @param a the array to print
     */
    public <T> void printArray(T[] a)
    {
        for(T i:a)
            System.out.print(i+" ");
    }

    /**
     * Prints out the tree
     * @param n
     */
    public void printTree(Node n)
    {
        System.out.println(n);
        for(Node a: n.childern)
            printTree(a);
    }
    /**
     * This runs the main program, first creating the categorization string
     * and then running the program with that tree
     **/
    public static void main(String args[])
    {
        //Create the category hierachy
        Prober prober=new Prober(args[3],args[0]);
        //prober.printTree(prober.rootNode);
        prober.runProber(Integer.parseInt(args[2]), Double.parseDouble(args[1]));
        //prober.runProber("diabetes.org", 100, .6);
        //"3k1.qprV34HoD2OYwrW5ZyZE44b78LIhUzTQLeXhCAUvafTLxHO8Ike75BszUFAp1mth7VQ"
    }
}